package SparkSQL

import org.apache.spark.sql.SparkSession

object SparkSQLMergeSchemaDemo {
  def main(args: Array[String]): Unit = {
    //创建或得到SparkSession
    val spark = SparkSession.builder()
      .appName("SparkSQLDataSource")
      .config("spark.sql.parquet.mergeSchema", true)
      .master("local[*]")
      .getOrCreate()


    //导入隐式转换
    import spark.implicits._

    //创建List集合，存储姓名和年龄
    val studentList = List(("jock", 22), ("lucy", 20))
    //将集合转为DataFrame，并指定列名为name和age
    val studentDF = spark.sparkContext
      .makeRDD(studentList)
      .toDF("name", "age")
    //将DataFrame写入HDFS的/students目录
//    studentDF.write.mode(SaveMode.Append)
//      .parquet("hdfs://hadoop101:8020/wh/students1")
    studentDF.show()
    /*+----+---+
    |name|age|
    +----+---+
    |jock| 22|
    |lucy| 20|
    +----+---+*/

    //创建List集合，存储姓名和成绩
    val studentList2 = List(("tom", 98), ("mary", 100))
    //将集合转为DataFrame，并指定列名为name和grade
    val studentDF2 = spark.sparkContext
      .makeRDD(studentList2)
      .toDF("name", "grade")
    //将DataFrame写入HDFS的/students目录（写入模式为Append）
//    studentDF2.write.mode(SaveMode.Append)
//      .parquet("hdfs://hadoop101:8020/wh/students1")
    studentDF2.show()
    /*+----+-----+
    |name|grade|
    +----+-----+
    | tom|   98|
    |mary|  100|
    +----+-----+*/

    //读取HDFS目录/students的Parquet文件数据，且合并Schema
    val mergedDF = spark.read.option("mergeSchema", "true")
      .parquet("hdfs://hadoop101:8020/wh/students1")
    //输出Schema信息
    mergedDF.printSchema()
    //输出数据内容
    mergedDF.show()
//    mergedDF.write.

//    // 将 DataFrame 转换成 JDBC 数据源，并写入 MySQL 数据库
//    //Spark Shell
//    mergedDF.write
//      .mode(SaveMode.Append)
//      .format("jdbc")
//      .option("url", "jdbc:mysql://localhost:13306/lindashixi?serverTimezone=GMT")
//      .option("driver", "com.mysql.jdbc.Driver")
//      .option("dbtable", "student")
//      .option("user", "root")
//      .option("password", "220724")
//      .save()

    // 关闭SparkSession
    spark.stop()
  }
  /*+----+-----+----+
  |name|grade| age|
  +----+-----+----+
  |mary|  100|null|
  | tom|   98|null|
  |jock| null|  22|
  |lucy| null|  20|
  +----+-----+----+*/
}
